from py2neo import Node, Relationship, Graph, NodeMatcher, RelationshipMatcher, Subgraph
import numpy as np
import json
import os
from repo_crawler import *
class DataBase:

    def __init__(self):
        self.graph = Graph("http://localhost:7474", password="xy001005")
        self.nodes = NodeMatcher(self.graph)
        self.relations = RelationshipMatcher(self.graph)

    # 删除图的所有内容
    def delete_all(self):
        self.graph.delete_all()

    # 查找所有符合给定标签的结点
    def print_nodes(self, tag):
        node_tag = self.nodes.match(tag).all()
        print("%s类的%s个节点如下：" % (tag, len(node_tag)))
        for node in node_tag:
            print(node)

    # 查找所有符合给定标签的关系
    def print_relations(self, tag):
        relation_tag = self.relations.match(tag).all()
        print("%s类的%s个关系如下：" % (tag, len(relation_tag)))
        for node in relation_tag:
            print(node)

    # 通过id查找包
    def match_pkg_by_id(self, id):
        res = self.nodes.match('Package', ID=id)
        return res.first()

    # 通过name查找repo
    def match_repo_by_name(self, name):
        res = self.nodes.match('Repository', name=name)
        return res.first()

    # 通过name查找developer
    def match_developer_by_name(self, name):
        res = self.nodes.match('developer', name=name)
        return res.first()

    # 通过name查找package
    def match_pkg_by_name(self, name):
        res = self.nodes.match('Package', name=name)
        return res.first()

    # 添加仓库信息结点
    def add_repos(self):
        count = 1
        json_names = []
        for _, _, fs in os.walk("./detail"):
            json_names = fs

        for json_name in json_names:
            with open("./detail/" + json_name, 'r', encoding='utf8')as fp:
                repo_data = json.load(fp)
                contributor = "https://gitee.com/api/v5/repos/src-openEuler/"+ repo_data['name'] +"/contributors"
                self.graph.create(Node('Repository', name=repo_data['name'],URL=repo_data["html_url"],language=repo_data["language"],owner=repo_data["owner"]["id"],contributor=contributor,stargazers_count=repo_data["stargazers_count"],member=repo_data["members"]))
            if count % 500 == 0:
                print("already save %s nodes" % count)
            count += 1

    # 添加commit结点（并添加其与仓库的关系）
    def add_commits(self):
        count = 1
        json_names = []
        for _, _, fs in os.walk("./commits"):
            json_names = fs

        for json_name in json_names:
            repo_name = json_name.split('_commits')[0]
            node1 = self.match_repo_by_name(repo_name)
            with open("./commits/" + json_name, 'r', encoding='utf8')as fp:
                commit_datas = json.load(fp)
                for commit_data in commit_datas:
                    if commit_data["committer"] is None :
                        continue
                    node2 = Node('commit', commit_message=commit_data["commit"]["message"],URL=commit_data["url"], developer_ID=commit_data["committer"]["id"])
                    self.graph.create(node2)
                    self.graph.create(Relationship(node2, "commit_update_repo", node1))
            if count % 100 == 0:
                print("already save %s commits" % count)
            count += 1

    # 添加issues结点（并添加其与仓库的关系）
    def add_issues(self):
        count = 1
        json_names = []
        for _, _, fs in os.walk("./issues"):
            json_names = fs

        for json_name in json_names:
            repo_name = json_name.split('_issues')[0]
            node1 = self.match_repo_by_name(repo_name)
            with open("./issues/" + json_name, 'r', encoding='utf8')as fp:
                issue_datas = json.load(fp)
                for issue_data in issue_datas:
                    if issue_data["finished_at"] is None:
                        finish_time = ""
                    else:
                        finish_time = issue_data["finished_at"]
                    if issue_data["updated_at"] is None:
                        update_time = ""
                    else:
                        update_time = issue_data["updated_at"]
                    if issue_data["deadline"] is None:
                        deadline = ""
                    else:
                        deadline = issue_data["deadline"]
                    comments = "https://gitee.com/api/v5/repos/src-openEuler/%s/issues/%s/comments?page=1&per_page=100&order=asc" %(repo_name,issue_data["number"])
                    node2 = Node('issue', ID=issue_data["id"],title=issue_data["title"],body=issue_data["body"],
                                 developer_ID=issue_data["user"]["id"],developer_url=issue_data["user"]["url"],
                                 type=issue_data["issue_type"],state=issue_data["issue_state"],priority=issue_data["priority"],
                                 create_time=issue_data["created_at"],update_time=update_time,finish_time=finish_time,deadline=deadline,
                                 comments=comments)
                    self.graph.create(node2)
                    self.graph.create(Relationship(node1, "issue_in_repo", node2))
            if count % 100 == 0:
                print("already save %s issues" % count)
            count += 1

    # 添加开发者结点（并添加其与仓库的关系）
    def add_developer(self):
        name_set = set()   #开发者名称集合，添加过的可以不用再次添加
        count = 1
        json_names = []
        for _, _, fs in os.walk("./users"):
            json_names = fs

        for json_name in json_names:
            repo_name = json_name.split('_users')[0]
            node1 = self.match_repo_by_name(repo_name)
            with open("./users/" + json_name, 'r', encoding='utf8')as fp:
                user_datas = json.load(fp)
                for developer in user_datas:
                    if developer["name"] in name_set:
                        node2 = self.match_developer_by_name(developer["name"])
                    else:
                        name_set.add(developer["name"])
                        contribution = [developer["stared"],developer["watched"],developer["followers"],developer["following"]]
                        node2 = Node('developer', name=developer["name"], URL=developer["html_url"], email=developer["email"],
                                     interest="To be selected", score=10,
                                     strengths="To be selected", style=1,
                                     prefer="To be selected",
                                     address="To be selected", experience=100, quality=100,
                                     activity=developer["events_url"],
                                     contribution=contribution)
                        self.graph.create(node2)
                    self.graph.create(Relationship(node2, "developer of", node1))
            if count % 100 == 0:
                print("already save %s developer" % count)
            count += 1

    # 添加包和仓库关系
    def add_rel_repo_pkg(self):
        count = 1
        pkg_name = get_pkg_name()
        for pkg in pkg_name:
            node1 = self.match_repo_by_name(pkg)
            node2 = self.match_pkg_by_name(pkg)   #store_at
            self.graph.create(Relationship(node2, "store_at", node1))
            if count % 100 == 0:
                print("already save %s rel" % count)
            count += 1

    #添加CVE漏洞数据
    def add_vulnerability(self):
        count = 1
        json_names = []
        for _, _, fs in os.walk("./CVE"):
            json_names = fs

        for json_name in json_names:
            with open("./CVE/" + json_name, 'r', encoding='utf8')as fp:
                json_datas = json.load(fp)
                for key, json_data in json_datas.items():
                    for CVE in json_data:
                        node1 = Node('Vulnerability', ID=CVE["id"],CVE_ID=CVE['cveId'], info=CVE["summary"],
                                     package=CVE["packageName"], status=CVE["status"],
                                     announcement_time=CVE["announcementTime"],score=CVE["cvsssCoreOE"])
                        self.graph.create(node1)
                        node2 = self.match_repo_by_name(CVE["packageName"])
                        if node2 is not None:
                            self.graph.create(Relationship(node1, "vul_exploit", node2))
                        if count % 100 == 0:
                            print("already save %s vul" % count)
                        count += 1


    #添加软件包之间的buildrequire（构建时依赖）
    def add_buildrequire(self,filename='buildre-v2.csv'):
        #filename = 'buildre-v2.csv'
        data = []
        rows = []
        with open(filename) as csvfile:
            csv_reader = csv.reader(csvfile)  # 使用csv.reader读取csvfile中的文件
            # header = next(csv_reader)        # 读取第一行每一列的标题
            for row in csv_reader:
                data.append(row[0])  # 选择某一列加入到data数组中
                while '' in row:  # 判断是否有空值在列表中
                    row.remove('')  # 如果有就直接通过remove删除
                new_row = []
                for i in row:
                    if i not in new_row:
                        new_row.append(i)
                rows.append(new_row)

        test_graph = Graph('http://localhost:7474', auth=('neo4j', 'xy001005'))  # 连接neo4j 数据库

        for package_br in rows:
            if len(package_br) == 1:
                continue
            else:
                for i in range(1, len(package_br)):
                    if package_br[i] in data:
                        print(package_br[0] + " yilaiyu " + package_br[i])
                        A = (test_graph.nodes.match('Package').where(name=package_br[0]).all())[0]
                        B = (test_graph.nodes.match('Package').where(name=package_br[i]).all())[0]
                        test_graph.create(Relationship(A, "buildrequire_pkg", B))


    # 添加软件包之间的require
    def add_require(self, filename='re-v2.csv'):
        data = []
        rows = []
        with open(filename) as csvfile:
            csv_reader = csv.reader(csvfile)  # 使用csv.reader读取csvfile中的文件
            # header = next(csv_reader)        # 读取第一行每一列的标题
            for row in csv_reader:
                data.append(row[0])  # 选择某一列加入到data数组中
                while '' in row:  # 判断是否有空值在列表中
                    row.remove('')  # 如果有就直接通过remove删除
                new_row = []
                for i in row:
                    if i not in new_row:
                        new_row.append(i)
                rows.append(new_row)

        test_graph = Graph('http://localhost:7474', auth=('neo4j', 'xy001005'))  # 连接neo4j 数据库

        for package_r in rows:
            if len(package_r) == 1:
                continue
            else:
                for i in range(1, len(package_r)):
                    if package_r[i] in data:
                        print(package_r[0] + " yilaiyu " + package_r[i])
                        A = (test_graph.nodes.match('Package').where(name=package_r[0]).all())[0]
                        B = (test_graph.nodes.match('Package').where(name=package_r[i]).all())[0]
                        test_graph.create(Relationship(A, "depends_on_pkg", B))

    #将包的数据添加到neo4j数据库中
    def add_package(self,filename="package_neo4j.csv"):
        count = 1
        with open(filename) as csvfile:
            csv_reader = csv.reader(csvfile)  # 使用csv.reader读取csvfile中的文件
            # header = next(csv_reader)        # 读取第一行每一列的标题
            for line in csv_reader:
                node = Node('Package', ID=line[7], name=line[1], isopen=line[2],
                             info=line[3], version=line[4],
                             release=line[5])
                self.graph.create(node)
                if count % 100 == 0:
                    print("already save %s Package" % count)
                count += 1



if __name__ == '__main__':
    db = DataBase()
    # db.delete_all()
    # db.add_repos()  #为图数据库添加仓库节点
    # db.add_commits() #为图数据库添加commit节点
    # db.add_issues()  #为图数据库添加issue节点
    # db.add_developer()
    # db.add_package()
    # db.add_buildrequire()
    # db.add_require()
    # db.add_rel_repo_pkg()
    # db.add_vulnerability()

